# Updated by Wei on 2013/07/20 night

from __future__ import division

def checkAuxFile():
    # Note
    # some useful functions I can use
    # inputProcessingDataFileHandler.tell()
    # inputProcessingDataFileHandler.seek()
    
    '''
    The following info needs to be verified
    1 1033 1723 500107
    2 8621 500183 4612332
    '''
    
    # This part is used for verification
    inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResults_head_10000_FOR_DEBUG"
    inputFileHandler = open(inputFileName,"r")
    
    inputFileHandler.seek(1723)
    for i in range(0,10):
        print inputFileHandler.readline().strip()
    
    '''
    inputFileHandler.seek(5691056)
    tryLine = inputFileHandler.readline()
    print tryLine,
    
    tryLine = inputFileHandler.readline()
    print tryLine,
    '''
    
    inputFileHandler.close()
    

def buildAuxFile():
    # This part is used for building the structure
    # The file dealing with here is:
    # inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/resultsNeededToAnalyze.txt"
    inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResults"
    # inputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResults_head_10000_FOR_DEBUG"
    inputFileHandler = open(inputFileName,"r")
    
    # in production mode
    outputFileName = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/rawResultsHead10KANDSemanticsTOP2MResultsAccessAuxFile_20130720"
    
    # in testing mode
    # outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/resultsNeededToAnalyzeAccessAuxFile_20130116_test.txt"
    
    # notes:
    # Updated by Wei 2013/01/16 afternoon
    # the file called: resultsNeededToAnalyze and the file called: resultsNeededToAnalyze.txt are generated in a very similar way. So the parser for generating the aux file for easy access should be similar
    outputFileHandler = open(outputFileName,"w")
    
    currentLine = inputFileHandler.readline()
    
    while currentLine:
        if currentLine.startswith("qid:"):
            currentInvertedIndexBeginningPosition = inputFileHandler.tell() - len(currentLine)
            print currentLine.strip()
            currentQueryQid = currentLine.strip().split(":")[1].strip()
            currentLine = inputFileHandler.readline()
            if currentLine.startswith("Search:"):
                # print currentLine.strip()
                # print currentLine.strip().split("Search:")
                queryContent = currentLine.strip().split("Search:")[1].strip() 
                print "queryContent:",queryContent
                
                # skip the line: [2013-07-13 16:24:23] 38871 terms with their lists length have been loaded.
                currentLine = inputFileHandler.readline()
                currentLine = inputFileHandler.readline()
                
                if currentLine.strip() != "" and not currentLine.startswith("(Need to update here 20130629)"):
                    
                    
                    
                    # this line should be the position index of each term if the IR system return some results
                    lineElements = currentLine.strip().split(" ")
                    
                     
                    print "lineElements:",lineElements
                    
                    currentQueryQueryTermPositionIndexDict = {}
                    
                    for element in lineElements:
                        currentQueryTerm = element.strip().split(":")[0]
                        currentQueryTermPositionIndex = int( element.strip().split(":")[1] )
                        if currentQueryTerm not in currentQueryQueryTermPositionIndexDict:
                            currentQueryQueryTermPositionIndexDict[currentQueryTerm] = currentQueryTermPositionIndex
                    
                    print "currentQueryQueryTermPositionIndexDict:",currentQueryQueryTermPositionIndexDict
                
                    # skip the line: universal_threshold_socre_of_posting_based_on_probability_:0
                    nextLine = inputFileHandler.readline()
                    # skip the line: computation_method:1
                    nextLine = inputFileHandler.readline()
                    # skip the line: NEW Logistic Regression probability logic implementation by Wei 2013/06/19
                    nextLine = inputFileHandler.readline()
                    # load the right line
                    nextLine = inputFileHandler.readline()
                    
                    # print "nextLine:",nextLine
                    # print "len( nextLine.strip().split(' ')):",len( nextLine.strip().split(" ") )
                    # exit(1)
                    
                    if len( nextLine.strip().split(" ") ) == 65:
                        # There are some result entries here
                        
                        while not currentLine.startswith("(Need to update here 20130629)"):
                            previousLine = currentLine
                            currentLine = inputFileHandler.readline()
                        
                        currentQueryNumberOfResultsInThisCollection = currentLine.strip().split(" ")[5].strip().split(".")[0]
                        
                        currentInvertedIndexEndingPosition = inputFileHandler.tell()
                        outputLine = currentQueryQid + " " + str(currentQueryNumberOfResultsInThisCollection) + " " + str(currentInvertedIndexBeginningPosition) + " " + str(currentInvertedIndexEndingPosition)
                        outputFileHandler.write(outputLine + "\n")
                        print outputLine
                else:
                    # This query returns 0 results from the IR system
                    currentQueryNumberOfResultsInThisCollection = 0
                    currentInvertedIndexBeginningPosition = -1
                    currentInvertedIndexEndingPosition = -1
                    outputLine = currentQueryQid + " " + str(currentQueryNumberOfResultsInThisCollection) + " " + str(currentInvertedIndexBeginningPosition) + " " + str(currentInvertedIndexEndingPosition)
                    outputFileHandler.write(outputLine + "\n")
                    print outputLine
            else:
                print "currentLine:",currentLine
                print "input file format, ERROR"
            print
        
        currentLine = inputFileHandler.readline()
    
    inputFileHandler.close()
    outputFileHandler.close()

print "Program Begins..."
# step1:
buildAuxFile()

# step2:
# checkAuxFile()
print "Program Ends."

